{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['batches.meta', 'data_batch_1', 'data_batch_2', 'data_batch_3', 'data_batch_4', 'data_batch_5', 'readme.html', 'test_batch']\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "CIFAR_DIR = \"./cifar-10-batches-py\"\n",
    "print(os.listdir(CIFAR_DIR))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 3072)\n",
      "(50000,)\n",
      "(10000, 3072)\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "def load_data(filename):\n",
    "    \"\"\"read data from data file.\"\"\"\n",
    "    with open(filename, 'rb') as f:\n",
    "        data = pickle.load(f, encoding='bytes')\n",
    "        return data[b'data'], data[b'labels']\n",
    "\n",
    "# tensorflow.Dataset.\n",
    "class CifarData:\n",
    "    def __init__(self, filenames, need_shuffle):\n",
    "        all_data = []\n",
    "        all_labels = []\n",
    "        for filename in filenames:\n",
    "            data, labels = load_data(filename)\n",
    "            all_data.append(data)\n",
    "            all_labels.append(labels)\n",
    "        self._data = np.vstack(all_data)\n",
    "        self._data = self._data / 127.5 - 1\n",
    "        self._labels = np.hstack(all_labels)\n",
    "        print(self._data.shape)\n",
    "        print(self._labels.shape)\n",
    "        \n",
    "        self._num_examples = self._data.shape[0]\n",
    "        self._need_shuffle = need_shuffle\n",
    "        self._indicator = 0\n",
    "        if self._need_shuffle:\n",
    "            self._shuffle_data()\n",
    "            \n",
    "    def _shuffle_data(self):\n",
    "        # [0,1,2,3,4,5] -> [5,3,2,4,0,1]\n",
    "        p = np.random.permutation(self._num_examples)\n",
    "        self._data = self._data[p]\n",
    "        self._labels = self._labels[p]\n",
    "    \n",
    "    def next_batch(self, batch_size):\n",
    "        \"\"\"return batch_size examples as a batch.\"\"\"\n",
    "        end_indicator = self._indicator + batch_size\n",
    "        if end_indicator > self._num_examples:\n",
    "            if self._need_shuffle:\n",
    "                self._shuffle_data()\n",
    "                self._indicator = 0\n",
    "                end_indicator = batch_size\n",
    "            else:\n",
    "                raise Exception(\"have no more examples\")\n",
    "        if end_indicator > self._num_examples:\n",
    "            raise Exception(\"batch size is larger than all examples\")\n",
    "        batch_data = self._data[self._indicator: end_indicator]\n",
    "        batch_labels = self._labels[self._indicator: end_indicator]\n",
    "        self._indicator = end_indicator\n",
    "        return batch_data, batch_labels\n",
    "\n",
    "train_filenames = [os.path.join(CIFAR_DIR, 'data_batch_%d' % i) for i in range(1, 6)]\n",
    "test_filenames = [os.path.join(CIFAR_DIR, 'test_batch')]\n",
    "\n",
    "train_data = CifarData(train_filenames, True)\n",
    "test_data = CifarData(test_filenames, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "def inception_block(x, output_channel_for_each_path, name):\n",
    "    \"\"\"\n",
    "    inception block implementation\n",
    "    \"\"\"\n",
    "    \"\"\"\n",
    "    Args:\n",
    "    - x:\n",
    "    - output_channel_for_each_path: [10 ,20 ,5]\n",
    "    - name:\n",
    "    \"\"\"\n",
    "    \n",
    "    with tf.variable_scope(name):\n",
    "        conv1_1 = tf.layers.conv2d(x,\n",
    "                                  output_channel_for_each_path[0],\n",
    "                                  (1, 1),\n",
    "                                  strides=(1, 1),\n",
    "                                  padding='same',\n",
    "                                  activation=tf.nn.relu,\n",
    "                                  name='conv1_1')\n",
    "        \n",
    "        conv3_3 = tf.layers.conv2d(x,\n",
    "                                  output_channel_for_each_path[1],\n",
    "                                  (3, 3),\n",
    "                                  strides=(1, 1),\n",
    "                                  padding='same',\n",
    "                                  activation=tf.nn.relu,\n",
    "                                  name='conv3_3')\n",
    "        \n",
    "        conv5_5 = tf.layers.conv2d(x,\n",
    "                                  output_channel_for_each_path[1],\n",
    "                                  (5, 5),\n",
    "                                  strides=(1, 1),\n",
    "                                  padding='same',\n",
    "                                  activation=tf.nn.relu,\n",
    "                                  name='conv5_5')\n",
    "        \n",
    "        max_pooling = tf.layers.max_pooling2d(x,\n",
    "                                             (2, 2),\n",
    "                                             (2, 2),\n",
    "                                             name='max_pooling')\n",
    "\n",
    "        max_pooling_shape = max_pooling.get_shape().as_list()[1:]\n",
    "        input_shape = x.get_shape().as_list()[1:]\n",
    "        width_padding = (input_shape[0] - max_pooling_shape[0]) // 2\n",
    "        heigth_padding = (input_shape[1] - max_pooling_shape[1]) // 2\n",
    "        padded_pooling = tf.pad(max_pooling,\n",
    "                               [[0 ,0],\n",
    "                                [width_padding, width_padding],\n",
    "                                [heigth_padding, heigth_padding],\n",
    "                                [0, 0]])\n",
    "        \n",
    "        concat_layer = tf.concat([conv1_1, conv3_3, conv5_5, padded_pooling], axis=3)\n",
    "        return concat_layer\n",
    "        \n",
    "x = tf.placeholder(tf.float32, [None, 3072])\n",
    "y = tf.placeholder(tf.int64, [None])\n",
    "x_image = tf.reshape(x, [-1, 3, 32, 32])\n",
    "x_image = tf.transpose(x_image, perm=[0, 2, 3, 1])\n",
    "\n",
    "conv1 = tf.layers.conv2d(x_image, 32, (3, 3), padding='same', activation=tf.nn.relu, name='conv1')\n",
    "pooling1 = tf.layers.max_pooling2d(conv1,\n",
    "                                   (2, 2), # pooling size\n",
    "                                   (2, 2), # stride size\n",
    "                                   name='pool1')\n",
    "\n",
    "inception_2a = inception_block(pooling1,\n",
    "                               [16, 16, 16],\n",
    "                               name='inception_2a')\n",
    "\n",
    "inception_2b = inception_block(inception_2a,\n",
    "                               [16, 16, 16],\n",
    "                               name='inception_2b')\n",
    "\n",
    "pooling2 = tf.layers.max_pooling2d(inception_2b,\n",
    "                                   (2, 2), # pooling size\n",
    "                                   (2, 2), # stride size\n",
    "                                   name='pool2')\n",
    "\n",
    "inception_3a = inception_block(pooling2,\n",
    "                               [16, 16, 16],\n",
    "                               name='inception_3a')\n",
    "\n",
    "inception_3b = inception_block(inception_3a,\n",
    "                               [16, 16, 16],\n",
    "                               name='inception_3b')\n",
    "\n",
    "pooling3 = tf.layers.max_pooling2d(inception_3b,\n",
    "                                   (2, 2), # pooling size\n",
    "                                   (2, 2), # stride size\n",
    "                                   name='pool3')\n",
    "\n",
    "flatten_pooling3 = tf.contrib.layers.flatten(pooling3)\n",
    "y_ = tf.layers.dense(flatten_pooling3, 10)\n",
    "\n",
    "loss = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=y_)\n",
    "# y_ = softmax\n",
    "# y = y_one_hot\n",
    "# loss = ylogy_\n",
    "\n",
    "# indices\n",
    "predict = tf.argmax(y_, 1)\n",
    "\n",
    "# [1, 0, 1, 1, 1, 0, 0, 0]\n",
    "correct_prediction = tf.equal(predict, y)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float64))\n",
    "\n",
    "with tf.name_scope('train_op'):\n",
    "    train_op = tf.train.AdamOptimizer(1e-3).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] Step: 500, loss: 1.34183, acc: 0.60000\n",
      "[Train] Step: 1000, loss: 1.18848, acc: 0.50000\n",
      "[Train] Step: 1500, loss: 1.25551, acc: 0.55000\n",
      "[Train] Step: 2000, loss: 2.08680, acc: 0.35000\n",
      "[Train] Step: 2500, loss: 1.51886, acc: 0.50000\n",
      "[Train] Step: 3000, loss: 0.82693, acc: 0.65000\n",
      "[Train] Step: 3500, loss: 0.86880, acc: 0.70000\n",
      "[Train] Step: 4000, loss: 0.61570, acc: 0.75000\n",
      "[Train] Step: 4500, loss: 1.07646, acc: 0.55000\n",
      "[Train] Step: 5000, loss: 0.41941, acc: 0.85000\n",
      "(10000, 3072)\n",
      "(10000,)\n",
      "[Test ] Step: 5000, acc: 0.70350\n",
      "[Train] Step: 5500, loss: 1.32069, acc: 0.65000\n",
      "[Train] Step: 6000, loss: 0.53365, acc: 0.80000\n",
      "[Train] Step: 6500, loss: 0.86413, acc: 0.55000\n",
      "[Train] Step: 7000, loss: 0.43128, acc: 0.85000\n",
      "[Train] Step: 7500, loss: 0.20335, acc: 0.95000\n",
      "[Train] Step: 8000, loss: 0.44341, acc: 0.85000\n",
      "[Train] Step: 8500, loss: 0.53123, acc: 0.85000\n",
      "[Train] Step: 9000, loss: 1.15050, acc: 0.55000\n",
      "[Train] Step: 9500, loss: 0.84252, acc: 0.65000\n",
      "[Train] Step: 10000, loss: 0.61253, acc: 0.75000\n",
      "(10000, 3072)\n",
      "(10000,)\n",
      "[Test ] Step: 10000, acc: 0.73800\n"
     ]
    }
   ],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "batch_size = 20\n",
    "train_steps = 10000\n",
    "test_steps = 100\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for i in range(train_steps):\n",
    "        batch_data, batch_labels = train_data.next_batch(batch_size)\n",
    "        loss_val, acc_val, _ = sess.run(\n",
    "            [loss, accuracy, train_op],\n",
    "            feed_dict={\n",
    "                x: batch_data,\n",
    "                y: batch_labels})\n",
    "        if (i+1) % 500 == 0:\n",
    "            print('[Train] Step: %d, loss: %4.5f, acc: %4.5f' % (i+1, loss_val, acc_val))\n",
    "        if (i+1) % 5000 == 0:\n",
    "            test_data = CifarData(test_filenames, False)\n",
    "            all_test_acc_val = []\n",
    "            for j in range(test_steps):\n",
    "                test_batch_data, test_batch_labels \\\n",
    "                    = test_data.next_batch(batch_size)\n",
    "                test_acc_val = sess.run(\n",
    "                    [accuracy],\n",
    "                    feed_dict = {\n",
    "                        x: test_batch_data, \n",
    "                        y: test_batch_labels\n",
    "                    })\n",
    "                all_test_acc_val.append(test_acc_val)\n",
    "            test_acc = np.mean(all_test_acc_val)\n",
    "            print('[Test ] Step: %d, acc: %4.5f' % (i+1, test_acc))\n",
    "                \n",
    "                \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
